{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#! /usr/bin/env python\n",
    "\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os\n",
    "import time\n",
    "import datetime\n",
    "import data_helpers\n",
    "from text_cnn import TextCNN\n",
    "from tensorflow.contrib import learn\n",
    "from six.moves import xrange\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Parameters:\n",
      "ALLOW_SOFT_PLACEMENT=True\n",
      "BATCH_SIZE=64\n",
      "CHECKPOINT_EVERY=100\n",
      "DEV_SAMPLE_PERCENTAGE=0.1\n",
      "DROPOUT_KEEP_PROB=0.5\n",
      "EMBEDDING_DIM=128\n",
      "EVALUATE_EVERY=100\n",
      "FILTER_SIZES=3,4,5\n",
      "L2_REG_LAMBDA=0.0005\n",
      "LOG_DEVICE_PLACEMENT=False\n",
      "NEGATIVE_DATA_FILE=./data/rt-polaritydata/rt-polarity.neg\n",
      "NUM_CHECKPOINTS=5\n",
      "NUM_EPOCHS=6\n",
      "NUM_FILTERS=64\n",
      "POSITIVE_DATA_FILE=./data/rt-polaritydata/rt-polarity.pos\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Parameters\n",
    "# ==================================================\n",
    "\n",
    "# Data loading params\n",
    "# validation数据集占比\n",
    "tf.flags.DEFINE_float(\"dev_sample_percentage\", .1, \"Percentage of the training data to use for validation\")\n",
    "# 正样本\n",
    "tf.flags.DEFINE_string(\"positive_data_file\", \"./data/rt-polaritydata/rt-polarity.pos\", \"Data source for the positive data.\")\n",
    "# 负样本\n",
    "tf.flags.DEFINE_string(\"negative_data_file\", \"./data/rt-polaritydata/rt-polarity.neg\", \"Data source for the negative data.\")\n",
    "\n",
    "# Model Hyperparameters\n",
    "# 词向量长度\n",
    "tf.flags.DEFINE_integer(\"embedding_dim\", 128, \"Dimensionality of character embedding (default: 128)\")\n",
    "# 卷积核大小\n",
    "tf.flags.DEFINE_string(\"filter_sizes\", \"3,4,5\", \"Comma-separated filter sizes (default: '3,4,5')\")\n",
    "# 每一种卷积核个数\n",
    "tf.flags.DEFINE_integer(\"num_filters\", 64, \"Number of filters per filter size (default: 128)\")\n",
    "# dropout参数\n",
    "tf.flags.DEFINE_float(\"dropout_keep_prob\", 0.5, \"Dropout keep probability (default: 0.5)\")\n",
    "# l2正则化参数\n",
    "tf.flags.DEFINE_float(\"l2_reg_lambda\", 0.0005, \"L2 regularization lambda (default: 0.0)\")\n",
    "\n",
    "# Training parameters\n",
    "# 批次大小\n",
    "tf.flags.DEFINE_integer(\"batch_size\", 64, \"Batch Size (default: 64)\")\n",
    "# 迭代周期\n",
    "tf.flags.DEFINE_integer(\"num_epochs\", 6, \"Number of training epochs (default: 200)\")\n",
    "# 多少step测试一次\n",
    "tf.flags.DEFINE_integer(\"evaluate_every\", 100, \"Evaluate model on dev set after this many steps (default: 100)\")\n",
    "# 多少step保存一次模型\n",
    "tf.flags.DEFINE_integer(\"checkpoint_every\", 100, \"Save model after this many steps (default: 100)\")\n",
    "# 最多保存多少个模型\n",
    "tf.flags.DEFINE_integer(\"num_checkpoints\", 5, \"Number of checkpoints to store (default: 5)\")\n",
    "# Misc Parameters\n",
    "# tensorFlow 会自动选择一个存在并且支持的设备来运行 operation\n",
    "tf.flags.DEFINE_boolean(\"allow_soft_placement\", True, \"Allow device soft device placement\")\n",
    "# 获取你的 operations 和 Tensor 被指派到哪个设备上运行\n",
    "tf.flags.DEFINE_boolean(\"log_device_placement\", False, \"Log placement of ops on devices\")\n",
    "\n",
    "# flags解析\n",
    "FLAGS = tf.flags.FLAGS\n",
    "FLAGS._parse_flags()\n",
    "\n",
    "# 打印所有参数\n",
    "print(\"\\nParameters:\")\n",
    "for attr, value in sorted(FLAGS.__flags.items()):\n",
    "    print(\"{}={}\".format(attr.upper(), value))\n",
    "print(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data...\n",
      "max_document_length: 56\n",
      "text: [ ['the', 'rock', 'is', 'destined', 'to', 'be', 'the', '21st', 'century', \"'s\", 'new', 'conan', 'and', 'that', 'he', \"'s\", 'going', 'to', 'make', 'a', 'splash', 'even', 'greater', 'than', 'arnold', 'schwarzenegger', ',', 'jean', 'claud', 'van', 'damme', 'or', 'steven', 'segal']\n",
      " ['the', 'gorgeously', 'elaborate', 'continuation', 'of', 'the', 'lord', 'of', 'the', 'rings', 'trilogy', 'is', 'so', 'huge', 'that', 'a', 'column', 'of', 'words', 'cannot', 'adequately', 'describe', 'co', 'writer', 'director', 'peter', 'jackson', \"'s\", 'expanded', 'vision', 'of', 'j', 'r', 'r', 'tolkien', \"'s\", 'middle', 'earth']\n",
      " ['effective', 'but', 'too', 'tepid', 'biopic']\n",
      " ['if', 'you', 'sometimes', 'like', 'to', 'go', 'to', 'the', 'movies', 'to', 'have', 'fun', ',', 'wasabi', 'is', 'a', 'good', 'place', 'to', 'start']\n",
      " ['emerges', 'as', 'something', 'rare', ',', 'an', 'issue', 'movie', 'that', \"'s\", 'so', 'honest', 'and', 'keenly', 'observed', 'that', 'it', 'does', \"n't\", 'feel', 'like', 'one']]\n"
     ]
    }
   ],
   "source": [
    "# Load data\n",
    "print(\"Loading data...\")\n",
    "# 读入预处理过后的数据和标签\n",
    "x_text, y = data_helpers.load_data_and_labels(FLAGS.positive_data_file, FLAGS.negative_data_file)\n",
    "\n",
    "# 一行数据最多的词汇数\n",
    "max_document_length = max([len(x.split(\" \")) for x in x_text])\n",
    "print(\"max_document_length:\",max_document_length)\n",
    "text = np.array([x.split(\" \") for x in x_text])\n",
    "print(\"text:\",text[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('w2v_dict.pickle', 'rb') as f:\n",
    "    w2v_dict = pickle.load(f)  \n",
    "\n",
    "x = []\n",
    "for line in text:\n",
    "    line_len = len(line)\n",
    "    text2num = []\n",
    "    for i in xrange(max_document_length):\n",
    "        if(i < line_len):\n",
    "            try:\n",
    "                text2num.append(w2v_dict[line[i]]) # 把词转为数字\n",
    "            except:\n",
    "                text2num.append(0) # 没有对应的词\n",
    "        else:\n",
    "            text2num.append(0) # 填充0\n",
    "    x.append(text2num)\n",
    "x = np.array(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_shape: (10662, 56)\n",
      "y_shape: (10662, 2)\n",
      "Train/Dev split: 9596/1066\n",
      "x: [[ 4281   594   149     2    52  6811     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0]\n",
      " [   35     0  1917    33   676    11   334     0    42    27  9203   149\n",
      "    779     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0]\n",
      " [   33    11 12217     4     2     1    52 36013     0 25600     3  9128\n",
      "  31463  1752   610 46365   677  3561    19     6  2205  2243     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0]\n",
      " [   61  3709  7453   726     0    72  2297 14128    14 11604     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0]\n",
      " [   72  5549     1  4318     2   384    54     0     5     1   121     2\n",
      "    158   432     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0]]\n",
      "y: [[1 0]\n",
      " [1 0]\n",
      " [1 0]\n",
      " [1 0]\n",
      " [1 0]]\n"
     ]
    }
   ],
   "source": [
    "print(\"x_shape:\",x.shape)\n",
    "print(\"y_shape:\",y.shape)\n",
    "\n",
    "# Randomly shuffle data\n",
    "np.random.seed(10)\n",
    "shuffle_indices = np.random.permutation(np.arange(len(y)))\n",
    "x_shuffled = x[shuffle_indices]\n",
    "y_shuffled = y[shuffle_indices]\n",
    "\n",
    "# Split train/test set\n",
    "# TODO: This is very crude, should use cross-validation\n",
    "# 数据集切分为两部分\n",
    "dev_sample_index = -1 * int(FLAGS.dev_sample_percentage * float(len(y)))\n",
    "x_train, x_dev = x_shuffled[:dev_sample_index], x_shuffled[dev_sample_index:]\n",
    "y_train, y_dev = y_shuffled[:dev_sample_index], y_shuffled[dev_sample_index:]\n",
    "print(\"Train/Dev split: {:d}/{:d}\".format(len(y_train), len(y_dev)))\n",
    "\n",
    "print(\"x:\",x_train[0:5])\n",
    "print(\"y:\",y_train[0:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Summary name embedding/W:0/grad/hist is illegal; using embedding/W_0/grad/hist instead.\n",
      "INFO:tensorflow:Summary name embedding/W:0/grad/sparsity is illegal; using embedding/W_0/grad/sparsity instead.\n",
      "INFO:tensorflow:Summary name conv-maxpool-3/W:0/grad/hist is illegal; using conv-maxpool-3/W_0/grad/hist instead.\n",
      "INFO:tensorflow:Summary name conv-maxpool-3/W:0/grad/sparsity is illegal; using conv-maxpool-3/W_0/grad/sparsity instead.\n",
      "INFO:tensorflow:Summary name conv-maxpool-3/b:0/grad/hist is illegal; using conv-maxpool-3/b_0/grad/hist instead.\n",
      "INFO:tensorflow:Summary name conv-maxpool-3/b:0/grad/sparsity is illegal; using conv-maxpool-3/b_0/grad/sparsity instead.\n",
      "INFO:tensorflow:Summary name conv-maxpool-4/W:0/grad/hist is illegal; using conv-maxpool-4/W_0/grad/hist instead.\n",
      "INFO:tensorflow:Summary name conv-maxpool-4/W:0/grad/sparsity is illegal; using conv-maxpool-4/W_0/grad/sparsity instead.\n",
      "INFO:tensorflow:Summary name conv-maxpool-4/b:0/grad/hist is illegal; using conv-maxpool-4/b_0/grad/hist instead.\n",
      "INFO:tensorflow:Summary name conv-maxpool-4/b:0/grad/sparsity is illegal; using conv-maxpool-4/b_0/grad/sparsity instead.\n",
      "INFO:tensorflow:Summary name conv-maxpool-5/W:0/grad/hist is illegal; using conv-maxpool-5/W_0/grad/hist instead.\n",
      "INFO:tensorflow:Summary name conv-maxpool-5/W:0/grad/sparsity is illegal; using conv-maxpool-5/W_0/grad/sparsity instead.\n",
      "INFO:tensorflow:Summary name conv-maxpool-5/b:0/grad/hist is illegal; using conv-maxpool-5/b_0/grad/hist instead.\n",
      "INFO:tensorflow:Summary name conv-maxpool-5/b:0/grad/sparsity is illegal; using conv-maxpool-5/b_0/grad/sparsity instead.\n",
      "INFO:tensorflow:Summary name W:0/grad/hist is illegal; using W_0/grad/hist instead.\n",
      "INFO:tensorflow:Summary name W:0/grad/sparsity is illegal; using W_0/grad/sparsity instead.\n",
      "INFO:tensorflow:Summary name output/b:0/grad/hist is illegal; using output/b_0/grad/hist instead.\n",
      "INFO:tensorflow:Summary name output/b:0/grad/sparsity is illegal; using output/b_0/grad/sparsity instead.\n",
      "Writing to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1499437902\n",
      "\n",
      "num_batches_per_epoch: 150\n",
      "2017-07-07T22:31:55.273542: step 10, loss 0.693351, acc 0.546875\n",
      "2017-07-07T22:31:57.210391: step 20, loss 0.720101, acc 0.453125\n",
      "2017-07-07T22:31:59.245975: step 30, loss 0.731953, acc 0.453125\n",
      "2017-07-07T22:32:01.237799: step 40, loss 0.675784, acc 0.53125\n",
      "2017-07-07T22:32:03.188958: step 50, loss 0.669614, acc 0.609375\n",
      "2017-07-07T22:32:05.173066: step 60, loss 0.703202, acc 0.53125\n",
      "2017-07-07T22:32:07.138543: step 70, loss 0.689708, acc 0.5625\n",
      "2017-07-07T22:32:09.195110: step 80, loss 0.677902, acc 0.546875\n",
      "2017-07-07T22:32:11.150315: step 90, loss 0.674923, acc 0.578125\n",
      "2017-07-07T22:32:13.230260: step 100, loss 0.659747, acc 0.625\n",
      "\n",
      "Evaluation:\n",
      "2017-07-07T22:32:13.393796: step 100, loss 0.670388, acc 0.610694\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1499437902\\checkpoints\\model-100\n",
      "\n",
      "2017-07-07T22:32:17.551560: step 110, loss 0.700677, acc 0.5625\n",
      "2017-07-07T22:32:19.604122: step 120, loss 0.648677, acc 0.640625\n",
      "2017-07-07T22:32:21.566224: step 130, loss 0.62302, acc 0.75\n",
      "2017-07-07T22:32:23.577420: step 140, loss 0.630056, acc 0.75\n",
      "2017-07-07T22:32:31.701846: step 150, loss 0.649838, acc 0.666667\n",
      "2017-07-07T22:32:33.884369: step 160, loss 0.651896, acc 0.640625\n",
      "2017-07-07T22:32:35.893218: step 170, loss 0.610358, acc 0.71875\n",
      "2017-07-07T22:32:37.889353: step 180, loss 0.60625, acc 0.734375\n",
      "2017-07-07T22:32:39.898906: step 190, loss 0.609266, acc 0.671875\n",
      "2017-07-07T22:32:41.865193: step 200, loss 0.584546, acc 0.71875\n",
      "\n",
      "Evaluation:\n",
      "2017-07-07T22:32:41.927667: step 200, loss 0.627007, acc 0.675422\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1499437902\\checkpoints\\model-200\n",
      "\n",
      "2017-07-07T22:32:45.874377: step 210, loss 0.611151, acc 0.6875\n",
      "2017-07-07T22:32:47.817325: step 220, loss 0.571888, acc 0.765625\n",
      "2017-07-07T22:32:49.776335: step 230, loss 0.548263, acc 0.84375\n",
      "2017-07-07T22:32:51.839956: step 240, loss 0.579348, acc 0.75\n",
      "2017-07-07T22:32:53.899767: step 250, loss 0.569048, acc 0.765625\n",
      "2017-07-07T22:32:55.862652: step 260, loss 0.52767, acc 0.8125\n",
      "2017-07-07T22:32:57.828652: step 270, loss 0.508189, acc 0.875\n",
      "2017-07-07T22:32:59.856962: step 280, loss 0.51305, acc 0.84375\n",
      "2017-07-07T22:33:01.841756: step 290, loss 0.552872, acc 0.734375\n",
      "2017-07-07T22:33:03.816281: step 300, loss 0.576546, acc 0.733333\n",
      "\n",
      "Evaluation:\n",
      "2017-07-07T22:33:03.878783: step 300, loss 0.575482, acc 0.736398\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1499437902\\checkpoints\\model-300\n",
      "\n",
      "2017-07-07T22:33:07.679490: step 310, loss 0.506772, acc 0.828125\n",
      "2017-07-07T22:33:09.660714: step 320, loss 0.502016, acc 0.8125\n",
      "2017-07-07T22:33:11.631695: step 330, loss 0.459141, acc 0.859375\n",
      "2017-07-07T22:33:13.615251: step 340, loss 0.484096, acc 0.828125\n",
      "2017-07-07T22:33:15.599586: step 350, loss 0.441231, acc 0.953125\n",
      "2017-07-07T22:33:17.560643: step 360, loss 0.437656, acc 0.953125\n",
      "2017-07-07T22:33:19.557463: step 370, loss 0.490475, acc 0.8125\n",
      "2017-07-07T22:33:21.609569: step 380, loss 0.432184, acc 0.921875\n",
      "2017-07-07T22:33:23.621027: step 390, loss 0.454882, acc 0.875\n",
      "2017-07-07T22:33:25.656083: step 400, loss 0.527213, acc 0.75\n",
      "\n",
      "Evaluation:\n",
      "2017-07-07T22:33:25.718558: step 400, loss 0.552731, acc 0.755159\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1499437902\\checkpoints\\model-400\n",
      "\n",
      "2017-07-07T22:33:29.707202: step 410, loss 0.446311, acc 0.90625\n",
      "2017-07-07T22:33:31.677522: step 420, loss 0.454056, acc 0.875\n",
      "2017-07-07T22:33:33.678075: step 430, loss 0.455128, acc 0.90625\n",
      "2017-07-07T22:33:35.647537: step 440, loss 0.488985, acc 0.84375\n",
      "2017-07-07T22:33:37.610882: step 450, loss 0.464398, acc 0.833333\n",
      "2017-07-07T22:33:39.580139: step 460, loss 0.41799, acc 0.9375\n",
      "2017-07-07T22:33:41.541500: step 470, loss 0.436185, acc 0.890625\n",
      "2017-07-07T22:33:43.614018: step 480, loss 0.386556, acc 0.96875\n",
      "2017-07-07T22:33:45.559685: step 490, loss 0.39684, acc 0.96875\n",
      "2017-07-07T22:33:47.569551: step 500, loss 0.375049, acc 0.953125\n",
      "\n",
      "Evaluation:\n",
      "2017-07-07T22:33:47.632026: step 500, loss 0.551021, acc 0.749531\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1499437902\\checkpoints\\model-500\n",
      "\n",
      "2017-07-07T22:33:51.572423: step 510, loss 0.389199, acc 0.9375\n",
      "2017-07-07T22:33:53.588026: step 520, loss 0.405881, acc 0.953125\n",
      "2017-07-07T22:33:55.578690: step 530, loss 0.406246, acc 0.921875\n",
      "2017-07-07T22:33:57.614611: step 540, loss 0.420449, acc 0.90625\n",
      "2017-07-07T22:33:59.597272: step 550, loss 0.420528, acc 0.90625\n",
      "2017-07-07T22:34:01.600168: step 560, loss 0.407758, acc 0.9375\n",
      "2017-07-07T22:34:03.574624: step 570, loss 0.38721, acc 0.9375\n",
      "2017-07-07T22:34:05.557455: step 580, loss 0.427144, acc 0.90625\n",
      "2017-07-07T22:34:07.546880: step 590, loss 0.406756, acc 0.9375\n",
      "2017-07-07T22:34:09.550471: step 600, loss 0.412875, acc 0.883333\n",
      "\n",
      "Evaluation:\n",
      "2017-07-07T22:34:09.628572: step 600, loss 0.550223, acc 0.747655\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1499437902\\checkpoints\\model-600\n",
      "\n",
      "2017-07-07T22:34:13.414748: step 610, loss 0.362709, acc 0.96875\n",
      "2017-07-07T22:34:15.408932: step 620, loss 0.395336, acc 0.921875\n",
      "2017-07-07T22:34:17.384613: step 630, loss 0.37343, acc 0.953125\n",
      "2017-07-07T22:34:19.390263: step 640, loss 0.352917, acc 1\n",
      "2017-07-07T22:34:21.374819: step 650, loss 0.356573, acc 0.953125\n",
      "2017-07-07T22:34:23.352797: step 660, loss 0.369842, acc 0.96875\n",
      "2017-07-07T22:34:25.379584: step 670, loss 0.367136, acc 0.953125\n",
      "2017-07-07T22:34:27.399438: step 680, loss 0.368675, acc 0.984375\n",
      "2017-07-07T22:34:29.489260: step 690, loss 0.373249, acc 0.953125\n",
      "2017-07-07T22:34:31.462713: step 700, loss 0.365361, acc 0.96875\n",
      "\n",
      "Evaluation:\n",
      "2017-07-07T22:34:31.525186: step 700, loss 0.550269, acc 0.743902\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1499437902\\checkpoints\\model-700\n",
      "\n",
      "2017-07-07T22:34:35.391575: step 710, loss 0.396324, acc 0.921875\n",
      "2017-07-07T22:34:37.377082: step 720, loss 0.35092, acc 0.984375\n",
      "2017-07-07T22:34:39.348222: step 730, loss 0.358922, acc 0.984375\n",
      "2017-07-07T22:34:41.342544: step 740, loss 0.349997, acc 0.96875\n",
      "2017-07-07T22:34:43.315707: step 750, loss 0.36803, acc 0.983333\n",
      "2017-07-07T22:34:45.293281: step 760, loss 0.350554, acc 0.96875\n",
      "2017-07-07T22:34:47.275175: step 770, loss 0.360621, acc 0.96875\n",
      "2017-07-07T22:34:49.493717: step 780, loss 0.331318, acc 1\n",
      "2017-07-07T22:34:51.658208: step 790, loss 0.340025, acc 0.984375\n",
      "2017-07-07T22:34:53.720702: step 800, loss 0.348142, acc 0.96875\n",
      "\n",
      "Evaluation:\n",
      "2017-07-07T22:34:53.783177: step 800, loss 0.54977, acc 0.746717\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1499437902\\checkpoints\\model-800\n",
      "\n",
      "2017-07-07T22:34:57.949378: step 810, loss 0.344458, acc 0.984375\n",
      "2017-07-07T22:35:00.136091: step 820, loss 0.341995, acc 1\n",
      "2017-07-07T22:35:02.243819: step 830, loss 0.370393, acc 0.953125\n",
      "2017-07-07T22:35:04.402424: step 840, loss 0.368348, acc 0.96875\n",
      "2017-07-07T22:35:06.506348: step 850, loss 0.352733, acc 0.984375\n",
      "2017-07-07T22:35:08.522172: step 860, loss 0.357023, acc 0.96875\n",
      "2017-07-07T22:35:10.550745: step 870, loss 0.369547, acc 0.953125\n",
      "2017-07-07T22:35:12.584415: step 880, loss 0.325835, acc 1\n",
      "2017-07-07T22:35:14.616288: step 890, loss 0.359149, acc 0.96875\n",
      "2017-07-07T22:35:16.742750: step 900, loss 0.377286, acc 0.95\n",
      "\n",
      "Evaluation:\n",
      "2017-07-07T22:35:16.805225: step 900, loss 0.550958, acc 0.750469\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1499437902\\checkpoints\\model-900\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Training\n",
    "# ==================================================\n",
    "\n",
    "with tf.Graph().as_default():\n",
    "    session_conf = tf.ConfigProto(\n",
    "      allow_soft_placement=FLAGS.allow_soft_placement,\n",
    "      log_device_placement=FLAGS.log_device_placement)\n",
    "    sess = tf.Session(config=session_conf)\n",
    "    with sess.as_default():\n",
    "        \n",
    "        cnn = TextCNN(\n",
    "            sequence_length=x_train.shape[1],\n",
    "            num_classes=y_train.shape[1],\n",
    "            vocab_size=len(w2v_dict),\n",
    "            embedding_size=FLAGS.embedding_dim,\n",
    "            filter_sizes=list(map(int, FLAGS.filter_sizes.split(\",\"))),\n",
    "            num_filters=FLAGS.num_filters,\n",
    "            l2_reg_lambda=FLAGS.l2_reg_lambda)\n",
    "\n",
    "        # Define Training procedure\n",
    "        global_step = tf.Variable(0, name=\"global_step\", trainable=False)\n",
    "        optimizer = tf.train.AdamOptimizer(1e-3)\n",
    "        # 计算梯度\n",
    "        grads_and_vars = optimizer.compute_gradients(cnn.loss)\n",
    "        # 将计算出的梯度应用到变量上，是函数minimize()的第二部分，\n",
    "        # 返回一个应用指定的梯度的操作Operation，对global_step做自增操作\n",
    "        train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step)\n",
    "\n",
    "        # Keep track of gradient values and sparsity (optional)\n",
    "        grad_summaries = []\n",
    "        for g, v in grads_and_vars:\n",
    "            if g is not None:\n",
    "                grad_hist_summary = tf.summary.histogram(\"{}/grad/hist\".format(v.name), g)\n",
    "                sparsity_summary = tf.summary.scalar(\"{}/grad/sparsity\".format(v.name), tf.nn.zero_fraction(g))\n",
    "                grad_summaries.append(grad_hist_summary)\n",
    "                grad_summaries.append(sparsity_summary)\n",
    "        grad_summaries_merged = tf.summary.merge(grad_summaries)\n",
    "\n",
    "        # Output directory for models and summaries\n",
    "        # 定义输出路径\n",
    "        timestamp = str(int(time.time()))\n",
    "        out_dir = os.path.abspath(os.path.join(os.path.curdir, \"runs\", timestamp))\n",
    "        print(\"Writing to {}\\n\".format(out_dir))\n",
    "\n",
    "        # Summaries for loss and accuracy\n",
    "        loss_summary = tf.summary.scalar(\"loss\", cnn.loss)\n",
    "        acc_summary = tf.summary.scalar(\"accuracy\", cnn.accuracy)\n",
    "\n",
    "        # Train Summaries\n",
    "        train_summary_op = tf.summary.merge([loss_summary, acc_summary, grad_summaries_merged])\n",
    "        train_summary_dir = os.path.join(out_dir, \"summaries\", \"train\")\n",
    "        train_summary_writer = tf.summary.FileWriter(train_summary_dir, sess.graph)\n",
    "\n",
    "        # Dev summaries\n",
    "        dev_summary_op = tf.summary.merge([loss_summary, acc_summary])\n",
    "        dev_summary_dir = os.path.join(out_dir, \"summaries\", \"dev\")\n",
    "        dev_summary_writer = tf.summary.FileWriter(dev_summary_dir, sess.graph)\n",
    "\n",
    "        # Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it\n",
    "        checkpoint_dir = os.path.abspath(os.path.join(out_dir, \"checkpoints\"))\n",
    "        checkpoint_prefix = os.path.join(checkpoint_dir, \"model\")\n",
    "        if not os.path.exists(checkpoint_dir):\n",
    "            os.makedirs(checkpoint_dir)\n",
    "        saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.num_checkpoints)\n",
    "\n",
    "        # Initialize all variables\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "\n",
    "        def train_step(x_batch, y_batch):\n",
    "            \"\"\"\n",
    "            A single training step\n",
    "            \"\"\"\n",
    "            feed_dict = {\n",
    "              cnn.input_x: x_batch,\n",
    "              cnn.input_y: y_batch,\n",
    "              cnn.dropout_keep_prob: FLAGS.dropout_keep_prob\n",
    "            }\n",
    "            _, step, summaries, loss, accuracy = sess.run(\n",
    "                [train_op, global_step, train_summary_op, cnn.loss, cnn.accuracy],\n",
    "                feed_dict)\n",
    "            time_str = datetime.datetime.now().isoformat()\n",
    "            if (step%10==0):\n",
    "                print(\"{}: step {}, loss {:g}, acc {:g}\".format(time_str, step, loss, accuracy))\n",
    "            train_summary_writer.add_summary(summaries, step)\n",
    "\n",
    "        def dev_step(x_batch, y_batch, writer=None):\n",
    "            \"\"\"\n",
    "            Evaluates model on a dev set\n",
    "            \"\"\"\n",
    "            feed_dict = {\n",
    "              cnn.input_x: x_batch,\n",
    "              cnn.input_y: y_batch,\n",
    "              cnn.dropout_keep_prob: 1.0\n",
    "            }\n",
    "            step, summaries, loss, accuracy = sess.run(\n",
    "                [global_step, dev_summary_op, cnn.loss, cnn.accuracy],\n",
    "                feed_dict)\n",
    "            time_str = datetime.datetime.now().isoformat()\n",
    "            print(\"{}: step {}, loss {:g}, acc {:g}\".format(time_str, step, loss, accuracy))\n",
    "            if writer:\n",
    "                writer.add_summary(summaries, step)\n",
    "\n",
    "        # Generate batches\n",
    "        batches = data_helpers.batch_iter(\n",
    "            list(zip(x_train, y_train)), FLAGS.batch_size, FLAGS.num_epochs)\n",
    "        # Training loop. For each batch...\n",
    "        for batch in batches:\n",
    "            x_batch, y_batch = zip(*batch)\n",
    "            train_step(x_batch, y_batch)\n",
    "            current_step = tf.train.global_step(sess, global_step)\n",
    "            # 测试\n",
    "            if current_step % FLAGS.evaluate_every == 0:\n",
    "                print(\"\\nEvaluation:\")\n",
    "                dev_step(x_dev, y_dev, writer=dev_summary_writer)\n",
    "                print(\"\")\n",
    "            # 保存模型\n",
    "            if current_step % FLAGS.checkpoint_every == 0:\n",
    "                path = saver.save(sess, checkpoint_prefix, global_step=current_step)\n",
    "                print(\"Saved model checkpoint to {}\\n\".format(path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
